{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of old_gadgets.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "ykrlKGVH3iGi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "df_train = pd.read_csv(\"Train.csv\")\n",
        "df_test = pd.read_csv(\"Test.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrCW22Xb5j8d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_feat = pd.DataFrame()\n",
        "df_feat_test = pd.DataFrame()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1OGsEyc5R8B",
        "colab_type": "code",
        "outputId": "f749d2a7-8ce9-4afe-e4a7-bf76a82481ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#gb\n",
        "text = df_train[\"Model_Info\"].values\n",
        "text2 = df_train[\"Additional_Description\"].values\n",
        "val = []\n",
        "\n",
        "for i in range(len(text)):\n",
        "  if '16gb' in text[i] or '16 gb' in text[i]:\n",
        "    val.append(1)\n",
        "  elif '32gb' in text[i] or '32 gb' in text[i]:\n",
        "    val.append(2)\n",
        "  elif '64gb' in text[i] or '64 gb' in text[i]:\n",
        "    val.append(3)\n",
        "  elif '128gb' in text[i] or '128 gb' in text[i]:\n",
        "    val.append(4)\n",
        "  elif '256gb' in text[i] or '256 gb' in text[i]:\n",
        "    val.append(5)\n",
        "  elif '512gb' in text[i] or '512 gb' in text[i]:\n",
        "    val.append(6)\n",
        "  else:\n",
        "    mx_gb = -1\n",
        "\n",
        "    if '512gb' in text2[i] or '512 gb' in text2[i]:\n",
        "      val.append(6)\n",
        "    elif '256gb' in text2[i] or '256 gb' in text2[i]:\n",
        "      val.append(5)\n",
        "    elif '128gb' in text2[i] or '128 gb' in text2[i]:\n",
        "      val.append(4)\n",
        "    elif '64gb' in text2[i] or '64 gb' in text2[i]:\n",
        "      val.append(3)\n",
        "    elif '32gb' in text2[i] or '32 gb' in text2[i]:\n",
        "      val.append(2)\n",
        "    elif '16gb' in text2[i] or '16 gb' in text2[i]:\n",
        "      val.append(1)\n",
        "    else:\n",
        "      val.append(0)\n",
        "\n",
        "    \n",
        "\n",
        "df_feat[\"size\"] = val\n",
        "\n",
        "\n",
        "#test\n",
        "\n",
        "text = df_test[\"Model_Info\"].values\n",
        "text2 = df_test[\"Additional_Description\"].values\n",
        "val = []\n",
        "\n",
        "for i in range(len(text)):\n",
        "  if '16gb' in text[i] or '16 gb' in text[i]:\n",
        "    val.append(1)\n",
        "  elif '32gb' in text[i] or '32 gb' in text[i]:\n",
        "    val.append(2)\n",
        "  elif '64gb' in text[i] or '64 gb' in text[i]:\n",
        "    val.append(3)\n",
        "  elif '128gb' in text[i] or '128 gb' in text[i]:\n",
        "    val.append(4)\n",
        "  elif '256gb' in text[i] or '256 gb' in text[i]:\n",
        "    val.append(5)\n",
        "  elif '512gb' in text[i] or '512 gb' in text[i]:\n",
        "    val.append(6)\n",
        "  else:\n",
        "    mx_gb = -1\n",
        "\n",
        "    if '512gb' in text2[i] or '512 gb' in text2[i]:\n",
        "      val.append(6)\n",
        "    elif '256gb' in text2[i] or '256 gb' in text2[i]:\n",
        "      val.append(5)\n",
        "    elif '128gb' in text2[i] or '128 gb' in text2[i]:\n",
        "      val.append(4)\n",
        "    elif '64gb' in text2[i] or '64 gb' in text2[i]:\n",
        "      val.append(3)\n",
        "    elif '32gb' in text2[i] or '32 gb' in text2[i]:\n",
        "      val.append(2)\n",
        "    elif '16gb' in text2[i] or '16 gb' in text2[i]:\n",
        "      val.append(1)\n",
        "    else:\n",
        "      val.append(0)\n",
        "\n",
        "    \n",
        "\n",
        "df_feat_test[\"size\"] = val"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2326\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCNdCCPb5gar",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_feat[\"Brand\"] = df_train[\"Brand\"]\n",
        "df_feat_test[\"Brand\"] = df_test[\"Brand\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ftd0vJ4Q5xwO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "df_feat[\"Locality\"] = df_train[\"Locality\"]\n",
        "df_feat_test[\"Locality\"] = df_test[\"Locality\"]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M21p8y1Z55ky",
        "colab_type": "code",
        "outputId": "e7ea3fec-c444-4a4b-d183-5c2d59eb548f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df_feat.head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>rom</th>\n",
              "      <th>Brand</th>\n",
              "      <th>price_mimic</th>\n",
              "      <th>Locality</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>11999</td>\n",
              "      <td>878</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>16901</td>\n",
              "      <td>1081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>256</td>\n",
              "      <td>1</td>\n",
              "      <td>34998</td>\n",
              "      <td>495</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>15999</td>\n",
              "      <td>287</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>16901</td>\n",
              "      <td>342</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   rom  Brand  price_mimic  Locality\n",
              "0   64      1        11999       878\n",
              "1    0      1        16901      1081\n",
              "2  256      1        34998       495\n",
              "3   64      1        15999       287\n",
              "4    0      1        16901       342"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTVLxGYt6GUv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def calc_iphone_value(brand_type, model_info):\n",
        "    iphone_value = list()\n",
        "    score = 0\n",
        "    for brand, text in zip(brand_type, model_info):\n",
        "        if brand == 1:\n",
        "          if (\"11\" in text or \"eleven\" in text or \"elven\" in text) and \"pro\" in text:\n",
        "              score = 28\n",
        "          elif \"11\" in text or (\"eleven\" in text or \"elven\" in text):\n",
        "              score = 27\n",
        "          elif \"xs\" in text and \"max\" in text:\n",
        "              score = 26\n",
        "          elif \"xs\" in text:\n",
        "              score = 25\n",
        "          elif \"x\" in text:\n",
        "              score = 24\n",
        "          elif (\"8s\" in text and \"plus\" in text) or (\"8\" in text and \"s\" in text and \"plus\" in text) or \"8splus\" in text:\n",
        "              score = 23\n",
        "          elif (\"8\" in text and \"plus\" in text) or \"8plus\" in text:\n",
        "              score = 22\n",
        "          elif (\"8\" in text and \"s\" in text) or \"8s\" in text:\n",
        "              score = 21\n",
        "          elif \"8\" in text or 'eight' in text:\n",
        "              score = 20\n",
        "          elif (\"7s\" in text and \"plus\" in text) or (\"7\" in text and \"s\" in text and \"plus\" in text) or \"7splus\" in text:\n",
        "              score = 19\n",
        "          elif (\"7\" in text and \"plus\" in text) or \"7plus\" in text:\n",
        "              score = 18\n",
        "          elif (\"7\" in text and \"s\" in text) or \"7s\" in text:\n",
        "              score = 17\n",
        "          elif \"7\" in text:\n",
        "              score = 16\n",
        "          elif (\"6s\" in text and \"plus\" in text) or (\"6\" in text and \"s\" in text and \"plus\" in text) or \"6splus\" in text:\n",
        "              score = 15\n",
        "          elif (\"6\" in text and \"plus\" in text) or \"6plus\" in text:\n",
        "              score = 14\n",
        "          elif (\"6\" in text and \"s\" in text) or \"6s\" in text:\n",
        "              score = 13\n",
        "          elif \"6\" in text:\n",
        "              score = 12\n",
        "          elif (\"5s\" in text and \"plus\" in text) or (\"5\" in text and \"s\" in text and \"plus\" in text) or \"5splus\" in text:\n",
        "              score = 11\n",
        "          elif (\"5\" in text and \"plus\" in text) or \"5plus\" in text:\n",
        "              score = 10\n",
        "          elif (\"5\" in text and \"s\" in text) or \"5s\" in text:\n",
        "              score = 9\n",
        "          elif (\"5\" in text and \"c\" in text) or \"5c\" in text:\n",
        "              score = 8\n",
        "          elif \"5\" in text:\n",
        "              score = 7\n",
        "          elif (\"4s\" in text and \"plus\" in text) or (\"4\" in text and \"s\" in text and \"plus\" in text) or \"4splus\" in text:\n",
        "              score = 6\n",
        "          elif (\"4\" in text and \"plus\" in text) or \"4plus\" in text:\n",
        "              score = 5\n",
        "          elif (\"4\" in text and \"s\" in text) or \"4s\" in text:\n",
        "              score = 4\n",
        "          elif (\"4\" in text and \"c\" in text) or \"4c\" in text:\n",
        "              score = 3\n",
        "          elif \"4\" in text:\n",
        "              score = 2\n",
        "          else:\n",
        "              score = 1\n",
        "        else:\n",
        "            score = 0\n",
        "        iphone_value.append(score)\n",
        "    \n",
        "    return iphone_value\n",
        "\n",
        "df_feat[\"iphone_value\"] = calc_iphone_value(df_train[\"Brand\"].values)\n",
        "df_feat_test[\"iphone_value\"] = calc_iphone_value(df_test[\"Brand\"].values)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z776lipReLIf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calc_honor_value(brand_type, model_info):\n",
        "    honor_value= list()\n",
        "    for brand, text in zip(brand_type, model_info):\n",
        "        score = 0\n",
        "        if brand == 0:\n",
        "            if \"9\" in text:\n",
        "                score =29\n",
        "            elif \"porsche\" in text:\n",
        "                score =28\n",
        "            elif (\"view\" in text and \"20\" in text) or \"view20\" in text:\n",
        "                score =27\n",
        "            elif \"10\" in text:\n",
        "                score =26\n",
        "            elif (\"view\" in text and \"10\" in text) or \"view10\" in text:\n",
        "                score =25\n",
        "            elif (\"8\" in text and \"pro\" in text) or \"8pro\" in text:\n",
        "                score =24\n",
        "            elif (\"nova\" in text and \"2\" in text and \"plus\" in text) or (\"nova\" in text and \"2plus\" in text) or (\"nova2\" in text and \"plus\" in text) or \"nova2plus\" in text:\n",
        "                score =23\n",
        "            elif \"20\" in text:\n",
        "                score =22\n",
        "            elif (\"nova\" in text and \"3\" in text and \"i\" in text) or (\"nova\" in text and \"3i\" in text) or (\"nova3\" in text and \"i\" in text) or \"nova3i\" in text:\n",
        "                score =21\n",
        "            elif \"8\" in text:\n",
        "                score =20\n",
        "            elif (\"970\" in text and \"i\" in text) or \"970i\" in text:\n",
        "                score =19\n",
        "            elif (\"970\" in text and \"i\" in text) or \"970i\" in text:\n",
        "                score =18\n",
        "            elif (\"p\" in text and \"20\" in text and \"lite\" in text) or (\"p\" in text and \"20lite\" in text) or (\"p20\" in text and \"lite\" in text) or \"p20lite\" in text:\n",
        "                score =17\n",
        "            elif (\"8\" in text and \"x\" in text) or \"8x\" in text:\n",
        "                score =16\n",
        "            elif (\"7\" in text and \"x\" in text) or \"7x\" in text:\n",
        "                score =15\n",
        "            elif (\"10\" in text and \"lite\" in text) or \"10lite\" in text:\n",
        "                score =14\n",
        "            elif (\"6\" in text and \"x\" in text) or \"6x\" in text:\n",
        "                score =13\n",
        "            elif (\"8\" in text and \"lite\" in text) or \"8lite\" in text:\n",
        "                score =12\n",
        "            elif (\"9\" in text and \"lite\" in text) or \"9lite\" in text:\n",
        "                score =11\n",
        "            elif (\"9\" in text and \"n\" in text) or \"9n\" in text:\n",
        "                score =10\n",
        "            elif (\"7\" in text and \"c\" in text) or \"7c\" in text:\n",
        "                score =9\n",
        "            elif (\"8\" in text and \"c\" in text) or \"8c\" in text:\n",
        "                score =8\n",
        "            elif (\"5\" in text and \"x\" in text) or \"5x\" in text:\n",
        "                score =7\n",
        "            elif (\"7\" in text and \"a\" in text) or \"7a\" in text:\n",
        "                score =6\n",
        "            elif (\"holly\" in text and \"4\" in text and \"plus\" in text) or (\"holly\" in text and \"4plus\" in text) or (\"holly4\" in text and \"plus\" in text) or \"holly4plus\" in text:\n",
        "                score =5\n",
        "            elif (\"4\" in text and \"x\" in text) or \"4x\" in text:\n",
        "                score =4\n",
        "            elif \"3\" in text:\n",
        "                score =3\n",
        "            elif \"g520\" in text:\n",
        "                score =2\n",
        "            else:\n",
        "                score =1\n",
        "        else:\n",
        "            score =0\n",
        "        honor_value.append(score)\n",
        "    return honor_value\n",
        "\n",
        "df_feat[\"honor_value\"] = calc_honor_value(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
        "df_feat_test[\"honor_value\"] = calc_honor_value(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wEXVt13z-C9r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calc_lg_value(brand_type, model_info):\n",
        "    lg_value= list()\n",
        "    for brand, text in zip(brand_type, model_info):\n",
        "        score = 0\n",
        "        if brand == 3:\n",
        "              if (\"v\" in text and \"50\" in text) or \"v50\" in text:\n",
        "                  score = 22\n",
        "              elif \"nexus\" in text:\n",
        "                  score = 21\n",
        "              elif \"g4\" in text or ((' 4 ' in text or ' 4' in text)):\n",
        "                  score = 20\n",
        "              elif \"v40\" in text:\n",
        "                  score = 19\n",
        "              elif \"thinq\" in text:\n",
        "                  score = 18\n",
        "              elif \"g6\" in text or ((' 6 ' in text or ' 6' in text)):\n",
        "                  score = 17\n",
        "              elif \"v30\" in text:\n",
        "                  score = 16\n",
        "              elif \"v20\" in text:\n",
        "                  score = 15\n",
        "              elif \"q7plus\" in text:\n",
        "                  score = 14\n",
        "              elif 'stylus 2' in text or 'stylush 2' in text or ('stylush' in text and (' 2 ' in text or ' 2' in text)):\n",
        "                  score = 13\n",
        "              elif \"stylush\" in text or 'stylus' in text:\n",
        "                  score = 12\n",
        "              elif \"q7\" in text or ((' 7 ' in text or ' 7' in text)):\n",
        "                  score = 11\n",
        "              elif \"k11\" in text or ((' 11 ' in text or ' 11' in text)):\n",
        "                  score = 10\n",
        "              elif \"g2\" in text or (' g ' in text and (' 2 ' in text or ' 2' in text)):\n",
        "                  score = 9\n",
        "              elif \"g5\" in text or ((' 5 ' in text or ' 5' in text)):\n",
        "                  score = 8\n",
        "              elif \"plus\" in text:\n",
        "                  score = 7\n",
        "              elif \"q6\" in text:\n",
        "                  score = 6\n",
        "              elif \"g3\" in text or ((' 3 ' in text or ' 3' in text)):\n",
        "                  score = 5\n",
        "              elif \"w30\" in text:\n",
        "                  score = 4\n",
        "              elif \"k10\" in text:\n",
        "                  score = 3\n",
        "              elif \"ph2\" in text:\n",
        "                  score = 2\n",
        "              else:\n",
        "                  score = 1\n",
        "\n",
        "        else:\n",
        "          score = 0\n",
        "        lg_value.append(score)\n",
        "\n",
        "    return lg_value\n",
        "\n",
        "df_feat[\"lg_value\"] = calc_lg_value(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
        "df_feat_test[\"lg_value\"] = calc_lg_value(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZGNXeTPnA_K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def calc_lenovo_value(brand_type, model_info):\n",
        "    lenovo_value= list()\n",
        "    for brand, text in zip(brand_type, model_info):\n",
        "        if brand == 2:\n",
        "            if \"k20\" in text:\n",
        "                score = 30\n",
        "            elif \"x2 lenovo\" in text or \"x2\" in text:\n",
        "                score = 29\n",
        "            elif \"lenovop2a42\" in text:\n",
        "                score = 28\n",
        "            elif \" p1 \" in text:\n",
        "                score = 27\n",
        "            elif \"zuk\" in text:\n",
        "                score = 26\n",
        "            elif \"k6 note\" in text or (\"k6\" in text and \"note\" in text) or 'k6note' in text or 'notek6' in text:\n",
        "                score = 25\n",
        "            elif \"a50\" in text:\n",
        "                score = 24  \n",
        "            elif \"k4 note\" in text or (\"k4\" in text and \"note\" in text) or 'k4note' in text or 'notek4' in text:\n",
        "                score = 23\n",
        "            elif \"vibe k5 note\" in text or (\"vibe\" in text and \"note\" in text and 'k5' in text):\n",
        "                score = 22\n",
        "            elif \"k8 note\" in text or (\"k8\" in text and \"note\" in text) or 'k8note' in text or 'notek8' in text:\n",
        "                score = 21\n",
        "            elif \"a20\" in text :\n",
        "                score = 20\n",
        "            elif \"z2 plus\" in text or (\"z2\" in text and \"plus\" in text):\n",
        "                score = 19\n",
        "            elif \"k5 note\" in text or (\"k5\" in text and \"note\" in text) or 'k5note' in text or 'notek5' in text:\n",
        "                score = 18\n",
        "            elif \"k6 power\" in text or (\"k6\" in text and \"power\" in text) or 'k6power' in text:\n",
        "                score = 17\n",
        "            elif \"name42tuxedo\" in text:\n",
        "                score = 16\n",
        "            elif \"phab 2 plus\" in text or (\"phab\" in text and '2' in text and 'plus' in text):\n",
        "                score = 15\n",
        "            elif \"vibe shot\" in text or (\"vibe\" in text and 'shot' in text):\n",
        "                score = 14\n",
        "            elif \"k8 plus\" in text or (\"k8\" in text and \"plus\" in text) or 'plusk8' in text:\n",
        "                score = 13\n",
        "            elif \"k8\" in text:\n",
        "                score = 12\n",
        "            elif \"vibe k5 plus\" in text or \"k5 plus\" in text or( \"k5\" in text and \"plus\" in text):\n",
        "                score = 11\n",
        "            elif \"13 mpl back camera 8 mpl front cam\" in text:\n",
        "                score = 10\n",
        "            elif \"a6600plus\" in text:\n",
        "                score = 9\n",
        "            elif \"z1\" in text:\n",
        "                score = 8\n",
        "            elif \"k3 note\" in text or (\"k3\" in text and \"note\" in text) or 'k3note' in text or 'notek3' in text:\n",
        "                score = 7\n",
        "            elif \"p1m40\" in text:\n",
        "                score = 6\n",
        "            elif \"p1m40\" in text or \"vibe p1m\" in text:\n",
        "                score = 5\n",
        "            elif \"k5\" in text or \"five\" in text:\n",
        "                score = 4\n",
        "            elif \"plus\" in text:\n",
        "                score = 3\n",
        "            elif \"a6000\" in text:\n",
        "                score = 2\n",
        "            else:\n",
        "                score = 1\n",
        "        else:\n",
        "            score = 0\n",
        "        lenovo_value.append(score)\n",
        "    return lenovo_value\n",
        "\n",
        "df_feat[\"lenovo_value\"] = calc_lenovo_value(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
        "df_feat_test[\"lenovo_value\"] = calc_lenovo_value(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1L1wDpGn7SGZ",
        "colab_type": "code",
        "outputId": "703371c3-746c-4409-8395-962c5fc5b7dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df_feat.head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>rom</th>\n",
              "      <th>Brand</th>\n",
              "      <th>Locality</th>\n",
              "      <th>iphone_type</th>\n",
              "      <th>iwatch_type</th>\n",
              "      <th>lg_type</th>\n",
              "      <th>honor_type</th>\n",
              "      <th>lenovo_type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>878</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1081</td>\n",
              "      <td>16</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>256</td>\n",
              "      <td>1</td>\n",
              "      <td>495</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>287</td>\n",
              "      <td>15</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>342</td>\n",
              "      <td>16</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   rom  Brand  Locality  ...  lg_type  honor_type  lenovo_type\n",
              "0   64      1       878  ...        0           0            0\n",
              "1    0      1      1081  ...        0           0            0\n",
              "2  256      1       495  ...        0           0            0\n",
              "3   64      1       287  ...        0           0            0\n",
              "4    0      1       342  ...        0           0            0\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEiRVWvC7l8w",
        "colab_type": "code",
        "outputId": "2369f474-ae11-4b8a-b890-42e067bbc6a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "pip install catboost"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting catboost\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b2/aa/e61819d04ef2bbee778bf4b3a748db1f3ad23512377e43ecfdc3211437a0/catboost-0.23.2-cp36-none-manylinux1_x86_64.whl (64.8MB)\n",
            "\u001b[K     |████████████████████████████████| 64.8MB 61kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.18.4)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.0.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.12.0)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.23.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTXpP1s-e3ch",
        "colab_type": "code",
        "outputId": "ad747281-c535-4203-b436-e346c0d57b94",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        }
      },
      "source": [
        "\n",
        "#idf + we\n",
        "\n",
        "import numpy as np\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from gensim.test.utils import common_texts, get_tmpfile\n",
        "from gensim.models import Word2Vec\n",
        "\n",
        "from gensim.models import Word2Vec\n",
        "from sklearn.decomposition import PCA\n",
        "from matplotlib import pyplot\n",
        "\n",
        "\n",
        "text = df_train[\"Model_Info\"].values\n",
        "text2  = df_test[\"Model_Info\"].values\n",
        "\n",
        "text = list(text) + list(text2)\n",
        "vectorizer = TfidfVectorizer(\n",
        "                        use_idf=True, # utiliza o idf como peso, fazendo tf*idf\n",
        "                        norm=None, # normaliza os vetores\n",
        "                        smooth_idf=False, #soma 1 ao N e ao ni => idf = ln(N+1 / ni+1)\n",
        "                        sublinear_tf=False, #tf = 1+ln(tf)\n",
        "                        binary=False,\n",
        "                        min_df=1,  max_features=None,\n",
        "                        strip_accents='unicode', # retira os acentos\n",
        "                        ngram_range=(1,2), preprocessor=None,stop_words=None, tokenizer=None, vocabulary=None\n",
        "             )\n",
        "X = vectorizer.fit_transform(text)\n",
        "idf = vectorizer.idf_\n",
        "p =  (dict(zip(vectorizer.get_feature_names(), idf)))\n",
        "\n",
        "print(p['iphone'])\n",
        "\n",
        "\n",
        "#word2vec\n",
        "a = df_train[\"Model_Info\"].values\n",
        "b = df_test[\"Model_Info\"].values\n",
        "c = df_train[\"Additional_Description\"].values\n",
        "d = df_test[\"Additional_Description\"].values\n",
        "\n",
        "print(a[:3])\n",
        "\n",
        "text = []\n",
        "\n",
        "vocab = []\n",
        "\n",
        "for i in range(len(a)):\n",
        "  z = a[i].strip().split(' ')\n",
        "  for j in range(len(z)):\n",
        "    if z[j] not in vocab:\n",
        "      vocab.append(z[j])\n",
        "  text.append(z)\n",
        "  #break\n",
        "\n",
        "for i in range(len(b)):\n",
        "  z = b[i].strip().split(' ')\n",
        "  for j in range(len(z)):\n",
        "    if z[j] not in vocab:\n",
        "      vocab.append(z[j])\n",
        "  text.append(z)\n",
        "  #break\n",
        "\n",
        "for i in range(len(c)):\n",
        "  z = c[i].strip().split(' ')\n",
        "  for j in range(len(z)):\n",
        "    if z[j] not in vocab:\n",
        "      vocab.append(z[j])\n",
        "  text.append(z)\n",
        "  #break\n",
        "\n",
        "for i in range(len(d)):\n",
        "  z = d[i].strip().split(' ')\n",
        "  for j in range(len(z)):\n",
        "    if z[j] not in vocab:\n",
        "      vocab.append(z[j])\n",
        "  text.append(z)\n",
        "  #break\n",
        "\n",
        "print(len(vocab))\n",
        "\n",
        "\n",
        "#path = get_tmpfile(\"word2vec.model\")\n",
        "model = Word2Vec(text,size=100, window=3, min_count=1, workers=4)\n",
        "model.train(text ,total_examples=len(text), epochs=50)\n",
        "print(model.wv['iphone'])\n",
        "\n",
        "model.wv.save('word.bin')\n",
        "\n",
        "print(model.similarity('iphone', 'honor'))\n",
        "\n",
        "\n",
        "sen = df_train[\"Model_Info\"].values\n",
        "\n",
        "\n",
        "vectorized = []\n",
        "for i in range(len(sen)):\n",
        "  z = sen[i].strip().split(' ')\n",
        "  vec = np.zeros(100)\n",
        "  for j in range(len(z)):\n",
        "    try:\n",
        "      vec += p[z[j]] * model.wv[z[j]]\n",
        "    except:\n",
        "      #print(z[j])\n",
        "      pass\n",
        "  vectorized.append(vec)\n",
        "  #l.append(len(z))\n",
        "\n",
        "print(len(vectorized))\n",
        "vectorized = np.array(vectorized)\n",
        "\n",
        "\n",
        "\n",
        "vectorized = vectorized.transpose()\n",
        "for i in range(100):\n",
        "  df_feat[\"feat_\"+str(i)] = vectorized[i]\n",
        "\n",
        "df_feat.head(5)\n",
        "\n",
        "\n",
        "#test part\n",
        "sen = df_train[\"Model_Info\"].values\n",
        "\n",
        "\n",
        "vectorized = []\n",
        "for i in range(len(sen)):\n",
        "  z = sen[i].strip().split(' ')\n",
        "  vec = np.zeros(100)\n",
        "  for j in range(len(z)):\n",
        "    try:\n",
        "      vec += p[z[j]] * model.wv[z[j]]\n",
        "    except:\n",
        "      #print(z[j])\n",
        "      pass\n",
        "  vectorized.append(vec)\n",
        "  #l.append(len(z))\n",
        "\n",
        "print(len(vectorized))\n",
        "vectorized = np.array(vectorized)\n",
        "\n",
        "\n",
        "\n",
        "vectorized = vectorized.transpose()\n",
        "for i in range(100):\n",
        "  df_feat[\"feat_\"+str(i)] = vectorized[i]\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#test\n",
        "\n",
        "\n",
        "\n",
        "sen = df_test[\"Model_Info\"].values\n",
        "\n",
        "\n",
        "vectorized = []\n",
        "for i in range(len(sen)):\n",
        "  z = sen[i].strip().split(' ')\n",
        "  vec = np.zeros(100)\n",
        "  for j in range(len(z)):\n",
        "    try:\n",
        "      vec += p[z[j]] * model.wv[z[j]]\n",
        "    except:\n",
        "      #print(z[j])\n",
        "      pass\n",
        "  vectorized.append(vec)\n",
        "  #l.append(len(z))\n",
        "\n",
        "print(len(vectorized))\n",
        "vectorized = np.array(vectorized)\n",
        "\n",
        "\n",
        "\n",
        "vectorized = vectorized.transpose()\n",
        "for i in range(100):\n",
        "  df_feat_test[\"feat_\"+str(i)] = vectorized[i]\n",
        "\n",
        "df_feat_test.head(5)\n",
        "\n",
        "\n",
        "#test part\n",
        "sen = df_test[\"Model_Info\"].values\n",
        "\n",
        "\n",
        "vectorized = []\n",
        "for i in range(len(sen)):\n",
        "  z = sen[i].strip().split(' ')\n",
        "  vec = np.zeros(100)\n",
        "  for j in range(len(z)):\n",
        "    try:\n",
        "      vec += p[z[j]] * model.wv[z[j]]\n",
        "    except:\n",
        "      #print(z[j])\n",
        "      pass\n",
        "  vectorized.append(vec)\n",
        "  #l.append(len(z))\n",
        "\n",
        "print(len(vectorized))\n",
        "vectorized = np.array(vectorized)\n",
        "\n",
        "\n",
        "\n",
        "vectorized = vectorized.transpose()\n",
        "for i in range(100):\n",
        "  df_feat_test[\"feat_\"+str(i)] = vectorized[i]\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.8338612471126554\n",
            "[' name0 name234 64gb space grey'\n",
            " ' phone 7 name42 name453 new condition box accessories'\n",
            " ' name0 x 256gb leess used good condition']\n",
            "7157\n",
            "[-1.4137496e-01  6.3632065e-01 -6.0834318e-01 -1.3777499e+00\n",
            " -2.0007653e+00 -3.0567793e-02 -3.0283895e-01  6.4354867e-01\n",
            " -7.4753565e-01  1.2167849e-01 -1.2483771e+00 -2.2853996e-01\n",
            " -5.6376201e-01 -3.2361639e-01  1.0240318e+00  3.4271100e-01\n",
            " -1.4612932e+00  9.1338855e-01  1.3492497e+00 -1.4805452e+00\n",
            "  1.1384077e+00 -2.1988873e-01  1.5021683e-02  1.4408296e-01\n",
            " -2.2608496e-01  1.0512000e+00  1.5627166e+00 -1.3344195e+00\n",
            "  1.9755366e+00 -6.8705308e-01 -1.5580155e+00 -1.5459311e+00\n",
            "  2.6260698e-01 -2.6422447e-01 -2.5806844e-03 -4.1897297e-01\n",
            "  7.0157886e-01 -1.3938203e+00 -3.2488966e+00 -2.9462728e-01\n",
            " -5.9173703e-01 -1.7720306e-01  1.9621883e-01 -2.8558370e-01\n",
            " -1.5346316e+00 -1.2094731e+00  1.3397609e-01  1.0717646e+00\n",
            " -5.8974024e-02  3.4988359e-01 -8.2910499e-03 -2.1040589e-01\n",
            "  8.5005343e-01  8.0089897e-02  3.3859128e-01 -1.2334322e+00\n",
            " -2.6139691e+00 -1.0147539e+00 -1.8892465e+00  5.5640566e-01\n",
            "  1.3204688e+00  8.7035882e-01 -1.1471624e+00 -5.8938700e-01\n",
            "  1.0532235e+00  8.7866396e-01 -5.2795272e-02  1.0285543e+00\n",
            "  1.6303922e+00  3.4255129e-01  5.1800603e-01  1.1021048e+00\n",
            " -3.1355536e-01  4.1157699e-01  6.3516718e-01  1.2706401e+00\n",
            " -2.0018058e+00 -1.8593061e+00 -9.8390482e-02  4.5287025e-01\n",
            " -2.7583179e-01  8.6091805e-01 -6.9317544e-01  4.3827450e-01\n",
            "  1.0813987e+00 -1.6729201e+00 -2.5084126e+00 -3.5515830e-02\n",
            "  1.0061120e+00 -1.9487320e-01  9.4315737e-01 -9.8816353e-01\n",
            "  9.0110642e-01  8.4193683e-01 -2.6623833e-01 -5.3055137e-01\n",
            " -1.5386428e+00  9.3733585e-01  2.7201819e+00  1.2035199e-01]\n",
            "0.03470749\n",
            "2326\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:89: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2326\n",
            "997\n",
            "997\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_o2lPpO5j4kK",
        "colab_type": "code",
        "outputId": "f83bb957-630b-4369-e544-d217cbd770d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "text = df_train[\"Model_Info\"].values\n",
        "val = []\n",
        "\n",
        "#old setting\n",
        "for i in range(len(text)):\n",
        "  if 'new' in text[i] or 'unused' in text[i] or 'sealed' in text[i]  or 'bill' in text[i] or 'warranty' in text[i] or 'waranty' in text[i] or 'working'in text[i]:\n",
        "    val.append(1)\n",
        "  else:\n",
        "    val.append(0)\n",
        "\n",
        "df_feat[\"good\"] = val\n",
        "\n",
        "#test\n",
        "\n",
        "text = df_test[\"Model_Info\"].values\n",
        "val = []\n",
        "\n",
        "for i in range(len(text)):\n",
        "  if 'new' in text[i] or 'unused' in text[i] or 'sealed' in text[i]   or 'working' in text[i] or 'bill' in text[i] or 'warranty' in text[i] or 'waranty' in text[i]:\n",
        "    val.append(1)\n",
        "  else:\n",
        "    val.append(0)\n",
        "\n",
        "df_feat_test[\"good\"] = val\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n#new setting\\ngood_words = [\\'scratchless\\',\\'cheap\\',\\'affordable\\',\\'never repaired\\',\\'great\\',\\'woratty\\',\\'classy\\',\\'latest\\',\\'letast\\',\\'packed\\', \\'genuine\\',\\'activ\\',\\'cod\\',\\'cash\\',\\'mast\\',\\'import\\',\\'exelant\\',\\'scratch less\\',\\'update\\',\\'offer\\',\\'awesome\\',\\'working\\',\\'best\\',\\'original\\',\\'issues solved\\',\\'excellent\\' , \\'neat\\' ,\\'good\\' ,\\'available\\' ,\\'new\\' ,\\'unused\\' , \\'sealed\\' , \\'bill\\' , \\'warranty\\' ,\\'waranty\\' ,\\'working\\']\\nbad_words = [\\'dead\\', \\'problem\\',\\'scratch\\',\\'broken\\',\\'break\\',\\'broke\\',\\'issue\\', \\'purani\\', \\'repair\\', \\'used\\', \\'spare\\', \\'spar\\', \\'crack\\',\\'nt\\', \\'not\\']\\ng = []\\nb = []\\n\\nfor i in range(len(text)):\\n  good_score = 0\\n  bad_score =  0\\n  for j in range(len(good_words)):  \\n    if good_words[j] in text[i]:\\n      good_score += 1\\n  for j in range(len(bad_words)):\\n    if bad_words[j] in text[i]:\\n      bad_score += 1\\n\\n  if good_score>0:\\n    g.append(1)\\n  else:\\n    g.append(0)\\n\\n  if bad_score>0:\\n    b.append(1)\\n  else:\\n    b.append(0)\\n  \\n\\n\\ndf_feat[\"good\"] = g\\n#df_feat[\"bad\"] = b\\n\\n#test\\n\\ntext = df_test[\"Model_Info\"].values\\ng = []\\nb = []\\n\\nfor i in range(len(text)):\\n  good_score = 0\\n  bad_score =  0\\n  for j in range(len(good_words)):  \\n    if good_words[j] in text[i]:\\n      good_score += 1\\n  for j in range(len(bad_words)):\\n    if bad_words[j] in text[i]:\\n      bad_score += 1\\n  if good_score>0:\\n    g.append(1)\\n  else:\\n    g.append(0)\\n\\n  if bad_score>0:\\n    b.append(1)\\n  else:\\n    b.append(0)\\n  \\n\\ndf_feat_test[\"good\"] = g\\n#df_feat_test[\"bad\"] = b\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TjiqaUQCkqyA",
        "colab_type": "code",
        "outputId": "08de6474-90da-4757-bc9a-049ccf60252b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "pip install catboost\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting catboost\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b2/aa/e61819d04ef2bbee778bf4b3a748db1f3ad23512377e43ecfdc3211437a0/catboost-0.23.2-cp36-none-manylinux1_x86_64.whl (64.8MB)\n",
            "\u001b[K     |████████████████████████████████| 64.8MB 60kB/s \n",
            "\u001b[?25hRequirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.2.1)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.18.4)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.0.4)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.12.0)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.2.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.23.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJY_adpD7zpf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from xgboost import XGBClassifier, XGBRegressor\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "import numpy as np\n",
        "from lightgbm import LGBMClassifier,LGBMRegressor\n",
        "from catboost import CatBoostClassifier,CatBoostRegressor\n",
        "from sklearn.metrics import mean_squared_log_error\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "\n",
        "\n",
        "text1 = df_train[\"Model_Info\"].values\n",
        "text2  = df_test[\"Model_Info\"].values\n",
        "\n",
        "text = list(text1) + list(text2)\n",
        "\n",
        "\n",
        "vectorizer = CountVectorizer(ngram_range=(1,2) ,min_df=6 )\n",
        "#X = vectorizer.fit_transform(text)\n",
        "\n",
        "labels = df_train[\"Price\"].values\n",
        "X_train, Y = df_feat.values, labels\n",
        "X_test = df_feat_test.values\n",
        "\n",
        "X_train = np.concatenate((X_train, vectorizer.fit_transform(text1).toarray()), axis=1)\n",
        "X_test = np.concatenate((X_test, vectorizer.transform(text2).toarray()), axis=1)\n",
        "\n",
        "\n",
        "kfold, scores = KFold(n_splits=5, shuffle=True, random_state=55), list()\n",
        "for training, testing in kfold.split(X_train):\n",
        "    x_train, x_test = X_train[training], X_train[testing]\n",
        "    y_train, y_test = np.log(Y[training]), Y[testing]\n",
        "    \n",
        " \n",
        "    model = XGBRegressor( random_state=57,verbose=500) \n",
        "    model.fit(x_train, y_train)\n",
        "    preds = model.predict(x_test)\n",
        "    score = np.sqrt(mean_squared_log_error( y_test, preds ))\n",
        "    scores.append(score)\n",
        "    print(score)\n",
        "print(\"Average: \", sum(scores)/len(scores))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbYcrIDT8RMk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = CatBoostRegressor( random_state=27,verbose=500)\n",
        "#model = CatBoostClassifier(random_state=27,  n_estimators=20, max_depth=4)\n",
        "model.fit(X_train, np.log(Y))\n",
        "preds = np.exp(model.predict(X_test))\n",
        "\n",
        "\n",
        "df_submit = pd.DataFrame({'Price': preds})\n",
        "df_submit.to_excel(\"submit.xlsx\", index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VJVogIPl74H2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#0.442079428174727 only count vect\n",
        "#0.43886961741781433 both tfidf   0.43506798495191024\n",
        "#0.4336968998976792 with good words(binary)\n",
        "# 0.43285217372434104 locality as int + above(we 100) 0.4314339048337291\n",
        "#0.4353848754335427 above + good + bad\n",
        "#0.4384639598700689 above - good\n",
        "#0.4375356303664656 good + bad both binary\n",
        "#0.4341520865905265 locality as cat + above\n",
        "#0.43786481204426153 with ram\n",
        "#0.5169049882440073 with specs and accessories\n",
        "#0..4347739453439021 with locality as int + we 150\n",
        "#0.4399352035917367 we 50\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}